import requests
from bs4 import BeautifulSoup
import pandas as pd
import concurrent.futures
import requests
import itertools
import networkx as nx
from collections import defaultdict
from functools import partial
from tqdm import tqdm
from requests_futures.sessions import FuturesSession
import matplotlib.pyplot as plt
import ast
import numpy as np
from statistics import mode
import json
from threading import Lock
import matplotlib.cm as cm
import pycountry
import re
import netwulf as nw
import matplotlib.patches as mpatches
import community.community_louvain as community_louvain
from IPython.display import Image, display, HTML
from nltk.corpus import stopwords
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures
from nltk.tokenize import word_tokenize, MWETokenizer
from nltk.stem import PorterStemmer
from collections import Counter
import math
from wordcloud import WordCloud
from nltk import bigrams
import collections
from scipy import stats
import seaborn as sns
from scipy.stats import pearsonr

headers = {"accept": "application/json"}
api_key = "f5813332cb558d374cbcb057ea2fc48b"

counter = 0
lock = Lock()

def movie_title_and_IDs_from_actor_ID(actor_id, session):
    global counter
    url = f"https://api.themoviedb.org/3/person/{actor_id}/movie_credits?api_key={api_key}"
    response = session.get(url)
    data = response.json()

    with lock:
        counter += 1
        if counter % 1000 == 0:
            print(f"Processed {counter} actors")
    # Return the whole movie dictionary, not just the title
    return [movie['title'] for movie in data['cast']], [movie['id'] for movie in data['cast']]

def actor_info_from_page(page, session):
    url = f"https://api.themoviedb.org/3/person/popular?api_key={api_key}&page={page}"
    response = session.get(url)
    data = response.json()
    people = []
    for person in data['results']:
        person_url = f"https://api.themoviedb.org/3/person/{person['id']}?api_key={api_key}"
        person_response = session.get(person_url)
        person_data = person_response.json()
        if person_data['deathday'] is not None:
            if person_data['deathday'] < '2010-01-01':
                continue
        people.append((person['name'], person['id'], person_data['gender'], person_data['birthday'], person_data['place_of_birth'], person_data['popularity']))
    return people

counter = 0
lock = Lock()

def fetch(session, url):
    global counter
    future = session.get(url, headers=headers)
    with lock:
        counter += 1
        if counter % 10000 == 0:
            print(f"Processed {counter} movies")
    
    return future

def movie_info_from_movie_ID(urls):
    with FuturesSession() as session:
        futures = [fetch(session, url) for url in urls]
        responses = [future.result().json() for future in tqdm(futures, total=len(futures))]
    return responses

with requests.Session() as session:
    session.headers.update(headers)
    with concurrent.futures.ThreadPoolExecutor() as executor:
        pages = list(range(1, 301))
        fetch_page_with_session = partial(actor_info_from_page, session=session)
        people = list(executor.map(fetch_page_with_session, pages))

actor_names, actor_ids, actor_genders, actor_birthdays, actor_birthplaces, actor_popularities = zip(*itertools.chain(*people))

# Create a DataFrame with 'actors' column
df_actors = pd.DataFrame(actor_names, columns=['actor'])

# Add 'ids', 'genders', and 'birthplaces' columns to the DataFrame
df_actors['actor_id'] = actor_ids
df_actors['gender'] = actor_genders
df_actors['age'] = actor_birthdays # This is not the age, but the birthday
df_actors['birthplace'] = actor_birthplaces
df_actors['popularity'] = actor_popularities

# Change birthplaces so that it only contains the country (text after the last comma)
df_actors['birthplace'] = df_actors['birthplace'].str.split(',').str[-1]

# Change birthday to age
df_actors['age'] = pd.to_datetime(df_actors['age'], errors='coerce')
df_actors['age'] = (pd.to_datetime('today') - df_actors['age']).dt.days // 365

# Drop rows with missing values
df_actors.dropna(inplace=True)

# Drop duplicates
df_actors.drop_duplicates(subset='actor_id', inplace=True)
#reset index
df_actors.reset_index(drop=True, inplace=True)

# Fetch movies for each actor and add them to 'movies' and 'movie_IDs' columns
with requests.Session() as session:
    session.headers.update(headers)
    movie_titles_and_ids_from_actor_ID_with_session = partial(movie_title_and_IDs_from_actor_ID, session=session)
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        movies_and_ids = list(executor.map(movie_titles_and_ids_from_actor_ID_with_session, df_actors['actor_id']))

# Add 'movies' column to df_actors
df_actors['movies'] = [x[0] for x in movies_and_ids]
df_actors['movie_IDs'] = [x[1] for x in movies_and_ids]

# Flatten movies_and_ids into two separate lists
movies = [item for sublist in [x[0] for x in movies_and_ids] for item in sublist]
ids = [item for sublist in [x[1] for x in movies_and_ids] for item in sublist]

# Convert the lists into a list of dictionaries
movies_and_ids_dict = [{'movie': movie, 'movie_ID': id} for movie, id in zip(movies, ids)]

# Convert the list of dictionaries into a DataFrame
df_movies = pd.DataFrame(movies_and_ids_dict)
df_movies = df_movies.drop_duplicates(subset='movie_ID')
df_movies.reset_index(drop=True, inplace=True)

# Prepare the URLs
urls = [f"https://api.themoviedb.org/3/movie/{id}?api_key={api_key}" for id in df_movies["movie_ID"]]
print('urls are prepared')

# Fetch all responses
responses = movie_info_from_movie_ID(urls)
print('all responses are fetched')

# Process the responses
for i, response in enumerate(responses):
    # if i % 100 == 0:
    #     print(f"Processing response {i+1}/{len(responses)}")
    if isinstance(response, Exception):
        print(f"Error: {response}")
        continue  # Skip this response
    # Process the response here

# Initialize empty lists to store the data
ratings = []
popularities = []
genres = []
release_dates = []
abstracts = []

# Process the responses one by one
for data in tqdm(responses):
    ratings.append(data.get('vote_average'))
    popularities.append(data.get('popularity'))
    genres.append([genre['name'] for genre in data.get('genres', [])])
    release_dates.append(data.get('release_date'))
    abstracts.append(data.get('overview'))

# Assign the lists to the DataFrame columns
df_movies['rating'] = ratings
df_movies['popularity'] = popularities
df_movies['genres'] = genres
df_movies['release_date'] = release_dates
df_movies['abstract'] = abstracts

# Drop duplicates
df_movies.drop_duplicates(subset='movie', inplace=True)

# Drop rows with missing values
df_movies.dropna(inplace=True)

# Remove rows with empty lists in 'genres' column
df_movies = df_movies[df_movies['genres'].apply(lambda x: len(x) > 0)]

# Remove rows with empty release dates
df_movies = df_movies[df_movies['release_date'].apply(lambda x: len(x) > 0)]

# Drop row if abstract is missing
df_movies = df_movies[df_movies['abstract'].apply(lambda x: len(x) > 0)]

# Drop row if rating is missing
df_movies = df_movies[df_movies['rating'].apply(lambda x: x > 0)]

# Drop row if popularity is missing
df_movies = df_movies[df_movies['popularity'].apply(lambda x: x > 0)]

# Reset index
df_movies.reset_index(drop=True, inplace=True)

# Combine the two dataframes so that each actor is associated with the movies they have acted in
df_actors_filtered = df_actors.copy()
df_movies_filtered = df_movies.copy()
df_movies_filtered.rename(columns={'movie_ID': 'movie_IDs'}, inplace=True)
df_movies_filtered.rename(columns={'popularity': 'movie_popularity'}, inplace=True)
df_actors_filtered.drop(columns=['movies'], inplace=True)

df_actors_filtered = df_actors_filtered.explode('movie_IDs').reset_index(drop=True)

df_actors_movies = df_actors_filtered.merge(df_movies_filtered, on='movie_IDs', how='inner')

# Remove all rows where the release date is before 2010 or after 2024 (to avoid movies that have not been released yet)
df_actors_movies = df_actors_movies[(df_actors_movies['release_date'] >= '2010-01-01') & (df_actors_movies['release_date'] <= '2024-05-08')]

# Collapse the the actors in the actor column so there is only one row per actor and the movies and movie_IDs are stored in lists
df_actors_filtered = df_actors_movies.groupby('actor').agg({'actor_id': 'first',
                                                             'gender': 'first',
                                                             'birthplace': 'first',
                                                             'age': 'first',
                                                             'popularity': 'first',
                                                             'movie': list, 
                                                             'movie_IDs': list}).reset_index()

df_movies_filtered = df_actors_movies.drop_duplicates(subset=['movie']).groupby('movie').agg({'movie_IDs': 'first',
                                                                                               'rating': 'first',
                                                                                               'popularity': 'first',
                                                                                               'genres': 'first',
                                                                                               'release_date': 'first',
                                                                                               'abstract': 'first'}).reset_index()

df_actors_filtered.rename(columns={'movie': 'movies'}, inplace=True)
df_movies_filtered.rename(columns={'movie_IDs': 'movie_ID'}, inplace=True)

# Remove actors with empty lists of movies
df_actors_filtered = df_actors_filtered[df_actors_filtered['movies'].apply(lambda x: len(x) > 0)]

# Clean birthplace column in actor dataframe
len_bef_clean = len(df_actors_filtered['birthplace'].unique())

country_names = [country.name for country in pycountry.countries]

def normalize_country_name(name):
    # strip name
    name = name.strip()
    # replace '.' with ''
    name = name.replace('.', '').replace(']', '')

    if "UK" in name or "İngiltere" in name:
        return "United Kingdom"
    try:
        # Try to get the country object
        country = pycountry.countries.get(name=name)
        if country is not None:
            # If the country object is found, return the official name
            return country.name
        else:
            # If the country object is not found, try to find it by its common name
            country = pycountry.countries.search_fuzzy(name)
            return country[0].name
    except LookupError:
        # Standardizing names
        for country_name in country_names:
            if country_name in name:
                name = country_name

        # Fixing abbreviations and wird instances
        if "Russia" in name:
            return "Russian Federation"
        elif "Türkiye" in name or "Turkey" in name:
            return "Türkiye"
        elif "USA" in name or " US" in name or "United States" in name:
            return "United States"
        elif "Korea" in name:
            return "Korea, Republic of"
        elif "Czech" in name:
            return "Czechia"
        # Hardcoded, could use package to translate, maybe not necessary, few occurences
        elif "TX" in name:
            return "United States"
        elif "Frankrike" in name:
            return "France"
        elif "Afrique du Sud" in name:
            return "South Africa"
        elif "Irlanda" in name:
            return "Ireland"
        elif "中国" in name or "中华民国" in name or "重庆" in name or "南京" in name:
            return "China"
        
        
        # Updating from old names
        if "now" in name:
            match = re.search(r'\[now (.*?)', name)
            if match:
                return match.group(1)

        # If the country is not found, return the original name
        return name

# Normalize the country names in the DataFrame
df_actors_filtered['birthplace'] = df_actors_filtered['birthplace'].apply(normalize_country_name)
df_actors_filtered = df_actors_filtered.dropna(subset=['birthplace'])

len_aft_clean = len(df_actors_filtered['birthplace'].unique())
# print("length before cleaning", len_bef_clean)
# print("length after cleaning", len_aft_clean)

# Function to find the genre counts and main genre of an actor
def find_genres_and_main_genre(movies_list, remove_drama=False): # nearly half of all movies have drama as genre, so not descriptive
    # Get the genres of the movies
    genres = [genre for movie in movies_list for genre in movie_genres.get(movie, [])]
    # Count the frequency of each genre
    genre_counts = pd.Series(genres).value_counts()
    # Remove the 'Drama' genre if specified
    if remove_drama and len(genre_counts) > 2:
        genre_counts.drop('Drama', errors='ignore', inplace=True)
        #genre_counts.drop('Thriller', errors='ignore', inplace=True)
    
    # Return the genre counts as a dictionary and the main genre
    return genre_counts.to_dict(), genre_counts.idxmax() if not genre_counts.empty else None

movie_genres = df_movies_filtered.set_index('movie')['genres'].to_dict()
# Apply the function to the 'movies' column of the actors DataFrame and create two new columns
df_actors_filtered['genres'], df_actors_filtered['main_genre'] = zip(*df_actors_filtered['movies'].apply(find_genres_and_main_genre, remove_drama=True))

# Function to find the average rating and popularity of an actor
def find_actor_rating(movies_list):
    # Get the ratings for each movie
    ratings = [movie_ratings[movie] for movie in movies_list if movie in movie_ratings]
    avg_rating = np.mean(ratings)

    return ratings, avg_rating

movie_ratings = df_movies_filtered.set_index('movie')['rating'].to_dict()

# Apply the function to the 'movies' column of the actors DataFrame and create two new columns
df_actors_filtered['ratings'], df_actors_filtered['avg_rating'] = \
        zip(*df_actors_filtered['movies'].apply(find_actor_rating))

# save the dataframes to csv files
df_actors_filtered.to_csv('data/actors.csv', index=False)
df_movies_filtered.to_csv('data/movies.csv', index=False)

# Function for correlation
def corr_func(x, y, **kwargs):
    r = pearsonr(x, y)[0]
    ax = plt.gca()
    size = np.sqrt(np.abs(r)) * 75  # Non-linear scaling for size
    color = 'cornflowerblue' if r > 0 else 'firebrick'  # Blue for positive, red for negative
    ax.annotate(f"{r:.2f}", xy=(.5, .5), xycoords=ax.transAxes, 
                size=max(size, 10),  # Minimum size to ensure readability
                ha='center', va='center', color=color)

# Function to label diagonal plots
def label_diag(x, **kwargs):
    ax = plt.gca()
    ax.set_title(x.name)  # Set the title to the variable name

df_actors_numeric = df_actors_filtered[['age', 'popularity', 'avg_rating']]

# Create an empty pairgrid with a custom color palette
sns.set(style="white", palette="husl")
grid = sns.PairGrid(df_actors_numeric)

# Map the plots to the lower, upper, and diagonal elements of the pairgrid
grid = grid.map_upper(corr_func)  # Annotate correlation coefficients
grid = grid.map_lower(sns.regplot, scatter_kws={'color':'blue','s': 10, 'alpha': 0.3}, line_kws={'color':'red','alpha': 0.5})
grid = grid.map_diag(sns.histplot, color='green', bins=45, stat="count")  # Histograms
grid = grid.map_diag(label_diag)  # Label the diagonal with variable names

# Remove axis labels for x and y axes
for ax in grid.axes.flatten():
    ax.set_xlabel('')
    ax.set_ylabel('')

# Save the plot with high DPI
#plt.savefig(fname="data/Numeric_actor_attributes.png", dpi=1000)
plt.suptitle("Numeric actor attributes")
plt.subplots_adjust(top=0.9)
plt.show()

df_movies_numeric = df_movies_filtered[['popularity', 'rating']]

# Create an empty pairgrid with a custom color palette
sns.set(style="white", palette="husl")
grid = sns.PairGrid(df_movies_numeric)

# Map the plots to the lower, upper, and diagonal elements of the pairgrid
grid = grid.map_upper(corr_func)  # Annotate correlation coefficients
grid = grid.map_lower(sns.regplot, scatter_kws={'color':'blue','s': 10, 'alpha': 0.3}, line_kws={'color':'red','alpha': 0.5})
grid = grid.map_diag(sns.histplot, color='green', bins=45)  # Histograms
grid = grid.map_diag(label_diag)  # Label the diagonal with variable names

# Remove axis labels for x and y axes
for ax in grid.axes.flatten():
    ax.set_xlabel('')
    ax.set_ylabel('')

# Save the plot with high DPI
#plt.savefig(fname="data/Numeric_movie_attributes.png", dpi=1000)
plt.suptitle("Numeric movie attributes")
plt.subplots_adjust(top=0.9)
plt.show()

# Set up the figure and axes in a 2x2 grid
fig, axs = plt.subplots(2, 2, figsize=(14, 12))

# First subplot: Top 10 Birthplaces of Actors
birthplace_counts = df_actors_filtered['birthplace'].value_counts()
birthplace_percentages = (birthplace_counts / df_actors_filtered['birthplace'].count()) * 100
top_5_birthplaces = birthplace_percentages[:10]
colors = cm.rainbow(np.linspace(0, 1, len(top_5_birthplaces)))
top_5_birthplaces.plot(kind='bar', color=colors, edgecolor='black', ax=axs[0, 0])
axs[0, 0].set_title('Top 10 Birthplaces of Actors')
axs[0, 0].set_xlabel('Birthplace')
axs[0, 0].set_ylabel('Percentage of Actors (%)')

# Second subplot: Top 10 Main Genres of Actors
main_genre_counts = df_actors_filtered['main_genre'].value_counts()
main_genre_percentages = (main_genre_counts / df_actors_filtered['main_genre'].count()) * 100
top_5_main_genres = main_genre_percentages[:10]
colors = cm.rainbow(np.linspace(0, 1, len(top_5_main_genres)))
top_5_main_genres.plot(kind='bar', color=colors, edgecolor='black', ax=axs[0, 1])
axs[0, 1].set_title('Top 10 Main Genres of Actors')
axs[0, 1].set_xlabel('Main Genre')
axs[0, 1].set_ylabel('Percentage of Actors (%)')

# Third subplot: Gender of Actors
gender_counts = df_actors_filtered['gender'].value_counts()
gender_percentages = (gender_counts / df_actors_filtered['gender'].count()) * 100
colors = cm.rainbow(np.linspace(0, 1, len(gender_counts)))
gender_percentages.plot(kind='bar', color=colors, edgecolor='black', ax=axs[1, 0])
axs[1, 0].set_title('Gender of Actors')
axs[1, 0].set_xlabel('Gender')
axs[1, 0].set_ylabel('Percentage of Actors (%)')
axs[1, 0].set_xticklabels(['Man', 'Woman', 'Other'])

# Remove the unused subplot
fig.delaxes(axs[1,1])

plt.tight_layout()
plt.show()

print("The number of unique birthplaces:",len(df_actors_filtered['birthplace'].unique()))
birthplaces_with_count_1 = birthplace_counts[birthplace_counts == 1]
print("Number of countries only associated with One actor in our dataset",len(birthplaces_with_count_1))

print("\nThe number of unique main genres:", len(df_actors_filtered['main_genre'].unique()))
main_genres_with_count_1 = main_genre_counts[main_genre_counts == 1]

The number of unique birthplaces: 115
Number of countries only associated with One actor in our dataset 42

The number of unique main genres: 19

# load the dataframes from csv files
df_actors_filtered = pd.read_csv('data/actors.csv', converters={'movies': ast.literal_eval, 'movie_IDs': ast.literal_eval, 'ratings': ast.literal_eval, 'genres': ast.literal_eval})
df_movies_filtered = pd.read_csv('data/movies.csv', converters={'genres': ast.literal_eval})

# Step 1: Prepare the Data
def create_network(df):
    movie_to_actors = defaultdict(list)

    for _ , row in df.iterrows():
        actor = row['actor']
        movies = row['movies']
        for movie in movies:
            movie_to_actors[movie].append(actor)

    edges = defaultdict(int)
    for actors in movie_to_actors.values():
        for pair in itertools.combinations(sorted(actors), 2):
            edges[pair] += 1

    # Step 2: Create the Graph
    G = nx.Graph()

    # Add nodes with attributes
    for _ , row in df.iterrows():
        G.add_node(row['actor'], gender=row['gender'], age=row['age'], birthplace=row['birthplace'], main_genre=row['main_genre'],
                avg_rating=row['avg_rating'], popularity=row['popularity'])

    # Add edges with weights
    for edge, weight in edges.items():

    # Adding only the edges with weight >= 2 to get less dense graph
        if weight >= 2:
            G.add_edge(*edge, weight=weight)

    return G

G = create_network(df_actors_filtered)

# save network to json()
data = nx.node_link_data(G)

# Save the dictionary as a JSON file
with open('data/actor_network.json', 'w') as f:
    json.dump(data, f)

# Find number of nodes and edges
N = G.number_of_nodes()
L = G.number_of_edges()
isolates = list(nx.isolates(G))
components = list(nx.connected_components(G))
largest_component = max(components, key=len)
edge_density = nx.density(G)

print("Number of nodes: ", N)
print("Number of edges: ", L)
print("Number of isolates: ", len(isolates))
print("Number of components: ", len(components))
print("Number of nodes in the largest component: ", len(largest_component))
print("Edge density: ", edge_density)

Number of nodes:  5100
Number of edges:  23736
Number of isolates:  1680
Number of components:  1714
Number of nodes in the largest component:  3341
Edge density:  0.0018255021169087365

# Compute node degrees
degrees = dict(G.degree())
weighted_degrees = dict(G.degree(weight='weight'))

# Compute the specificied values
average_degree = np.mean(list(degrees.values()))
median_degree = np.median(list(degrees.values()))
try:
    mode_degree = mode(list(degrees.values())) #Degree value that occurs with highest frequency among the nodes
except:
    mode_degree = "No unique mode"
min_degree = min(degrees.values())
max_degree = max(degrees.values())

# Same calculations but for STRENGTH (WEIGHTED DEGREE)
average_weighted_degree = np.mean(list(weighted_degrees.values()))
median_weighted_degree = np.median(list(weighted_degrees.values()))
try:
    mode_weighted_degree = mode(list(weighted_degrees.values()))
except:
    mode_weighted_degree = "No unique mode"
min_weighted_degree = min(weighted_degrees.values())
max_weighted_degree = max(weighted_degrees.values())

print("Degree Analysis:")
print(f"Average Degree: {average_degree}")
print(f"Median Degree: {median_degree}")
print(f"Mode Degree: {mode_degree}")
print(f"Minimum Degree: {min_degree}")
print(f"Maximum Degree: {max_degree}")

print("\nWeighted Degree Analysis:")
print(f"Average Weighted Degree: {average_weighted_degree}")
print(f"Median Weighted Degree: {median_weighted_degree}")
print(f"Mode Weighted Degree: {mode_weighted_degree}")
print(f"Minimum Weighted Degree: {min_weighted_degree}")
print(f"Maximum Weighted Degree: {max_weighted_degree}")

Degree Analysis:
Average Degree: 9.308235294117647
Median Degree: 2.0
Mode Degree: 0
Minimum Degree: 0
Maximum Degree: 184

Weighted Degree Analysis:
Average Weighted Degree: 25.587450980392155
Median Weighted Degree: 5.0
Mode Weighted Degree: 0
Minimum Weighted Degree: 0
Maximum Weighted Degree: 619

# Calculate degree distribution
degree_sequence = sorted([d for n, d in G.degree()], reverse=False)  # sort in ascending order
degree_count = nx.degree_histogram(G)

fig, axs = plt.subplots(1, 2, figsize=(16, 6))

# degree distribution
axs[0].bar(range(len(degree_count)), degree_count, align='center')
axs[0].axvline(average_degree, color='r', linestyle='-', label='Average Degree')
axs[0].set_xlabel('Degree')
axs[0].set_ylabel('Count')
axs[0].set_title('Degree Distribution')
axs[0].legend()

# loglog degree distribution
axs[1].bar(range(len(degree_count)), degree_count, align='center')
axs[1].set_yscale('log')
axs[1].set_xscale('log')
axs[1].set_xlabel('Degree')
axs[1].set_ylabel('Count')
axs[1].set_title('Degree Distribution (log scale)')
axs[1].axvline(average_degree, color='r', linestyle='-', label='Average Degree')
axs[1].legend()

plt.show()

# Sort the edges by weight in descending order
sorted_edges = sorted(G.edges.data(), key=lambda x: x[2]['weight'], reverse=True)
most_connected_actors = sorted(degrees.items(), key=lambda x: x[1], reverse=True)
most_connected_actors_weighted = sorted(weighted_degrees.items(), key=lambda x: x[1], reverse=True)

print("Top 5 actors with the highest degree:")
for i, actor in enumerate(most_connected_actors[:5], start=1):
    print(f"{i}: Degree: {actor[1]},\t ({G.nodes[actor[0]]['main_genre']})\t {actor[0]}")

print("\nTop 5 actors with the highest weighted degree:")
for i, actor in enumerate(most_connected_actors_weighted[:5], start=1):
    print(f"{i}: Degree: {actor[1]},\t ({G.nodes[actor[0]]['main_genre']})\t {actor[0]}")

print("\nTop 5 most important edges:")
for i, edge in enumerate(sorted_edges[:5], start=1):
    print(f"{i}: Degree: {edge[2]['weight']}, ({G.nodes[edge[0]]['main_genre']} - {G.nodes[edge[1]]['main_genre']}) \t {edge[0]} - {edge[1]}")

Top 5 actors with the highest degree:
1: Degree: 184,	 (Action)	 Samuel L. Jackson
2: Degree: 142,	 (Action)	 Stan Lee
3: Degree: 132,	 (Documentary)	 Benedict Cumberbatch
4: Degree: 131,	 (Comedy)	 Bill Hader
5: Degree: 129,	 (Animation)	 Fred Tatasciore

Top 5 actors with the highest weighted degree:
1: Degree: 619,	 (Animation)	 Grey DeLisle
2: Degree: 561,	 (Animation)	 Fred Tatasciore
3: Degree: 546,	 (Action)	 Samuel L. Jackson
4: Degree: 541,	 (Animation)	 Tara Strong
5: Degree: 492,	 (Animation)	 Hiroshi Kamiya

Top 5 most important edges:
1: Degree: 46, (Animation - Animation) 	 Frank Welker - Grey DeLisle
2: Degree: 42, (Action - Action) 	 Adam Copeland - John Cena
3: Degree: 36, (Animation - Animation) 	 Ikue Otani - Megumi Hayashibara
4: Degree: 35, (Action - Documentary) 	 John Cena - Michael Hickenbottom
5: Degree: 34, (Action - Action) 	 Dwayne Johnson - John Cena

# Compute the probability that makes the expected number of edges equal to the actual number of edges in the graph.
p = 2*L/(N*(N-1))
print("Prob:",p)

# Compute the natural logarithm of N
ln_N = np.log(N)

print(f"k: {average_degree:.2f} > ln(N): {ln_N:.2f}\np: {p:.5f} > ln(N)/N: {ln_N/N:.5f}")

Prob: 0.0018255021169087365
k: 9.31 > ln(N): 8.54
p: 0.00183 > ln(N)/N: 0.00167

# Plot with netwulf
G_plot_nation = G.copy()

for k, v in G_plot_nation.nodes(data=True):
    v['group'] = v['birthplace']; del v['birthplace']

network, config = nw.visualize(G_plot_nation)

# Find the color of all nodes in the network and find the top 10 most used colors
node_colors = [node['color'] for node in network['nodes']]

# Count the frequency of each color
color_counts = pd.Series(node_colors).value_counts()

# Get the top 10 most used colors
top_10_colors = color_counts.head(10)

# Create color dictionary
color_dict = {}

# Go through the nodes and assign the color to the birthplace
for color in top_10_colors.index:
    for node in network['nodes']:
        if node['color'] == color:
            color_dict[color] = df_actors_filtered[df_actors_filtered['actor']==node['id']]['birthplace'].values[0]
            break

# Print the color dictionary
for color, birthplace in color_dict.items():
    print(f"Color: {color}, Birthplace: {birthplace}")

# Creating a color legend for the countries
patches = [mpatches.Patch(color=color, label=f"{color_dict[color]}") for color in color_dict]

plt.figure(figsize=(5,2))
plt.legend(handles=patches, loc='center', frameon=False)
plt.axis('off')
plt.show()

fig,ax = nw.draw_netwulf(network)
plt.legend(handles=patches, loc='upper left', frameon=False)
fig.set_size_inches(12, 8)

# Save the figure
# fig.savefig('figures/country_graph.png')

# Load the saved png-file "country_graph.png" and plot it in the notebook
Image(filename='figures/country_graph.png')

# Calculate the degree assortativity coefficient
degree_assortativity = nx.degree_assortativity_coefficient(G)

# Calculate the attribute assortativity coefficient
country_assortativity = nx.attribute_assortativity_coefficient(G, 'birthplace')

gender_assortativity = nx.attribute_assortativity_coefficient(G, 'gender')

# Calculate the numeric assortativity coefficient
age_assortativity = nx.numeric_assortativity_coefficient(G, 'age')

# Calcualte the assortativity for main_genre
main_genre_assortativity = nx.attribute_assortativity_coefficient(G, 'main_genre')

rating_assortativity = nx.numeric_assortativity_coefficient(G, 'avg_rating')
popularity_assortativity = nx.numeric_assortativity_coefficient(G, 'popularity')

print(f"Categorical:\nGenre assortativity: {main_genre_assortativity}, \nBirth place assortativity: {country_assortativity}, \nGender assortativity: {gender_assortativity}")

print(f"\nNumeric:\nDegree assortativity: {degree_assortativity}, \nAge assortativity: {age_assortativity}, \nRating assortativity: {rating_assortativity}, \nPopularity assortativity: {popularity_assortativity}")

Categorical:
Genre assortativity: 0.33905127371077814, 
Birth place assortativity: 0.4761740970872584, 
Gender assortativity: 0.057871063989161226

Numeric:
Degree assortativity: 0.19237732816620495, 
Age assortativity: 0.3168726079482054, 
Rating assortativity: 0.4495447349115166, 
Popularity assortativity: 0.1557052667529273

# Calculate the closeness centrality of the network
closeness_centrality = nx.closeness_centrality(G)
# Calculate the eigenvector centrality of the network
eigenvector_centrality = nx.eigenvector_centrality(G)

# Sort the actors according to the closeness centrality
sorted_closeness_centrality = sorted(closeness_centrality.items(), key=lambda x: x[1], reverse=True)
# Sort the scientists according to the eigenvector centrality
sorted_eigenvector_centrality = sorted(eigenvector_centrality.items(), key=lambda x: x[1], reverse=True)

# Find the 10 most central actors
most_central_actors_closeness = sorted_closeness_centrality[:10]
# Find the 10 most central actors
most_central_actors_eigenvector = sorted_eigenvector_centrality[:10]

print(f'{"Actor (Closeness Centrality)":<30} {"Score":<10} | {"Actor (Eigenvector Centrality)":<30} {"Score"}')
print('-'*80)
for actor1, actor2 in zip(most_central_actors_closeness, most_central_actors_eigenvector):
    print(f'{actor1[0]:<30} {actor1[1]:<10.3f} | {actor2[0]:<30} {actor2[1]:.3f}')

Actor (Closeness Centrality)   Score      | Actor (Eigenvector Centrality) Score
--------------------------------------------------------------------------------
Samuel L. Jackson              0.236      | Miyuki Sawashiro               0.151
Stan Lee                       0.232      | Hiroshi Kamiya                 0.149
Fred Tatasciore                0.229      | Takahiro Sakurai               0.149
Benedict Cumberbatch           0.225      | Yuki Kaji                      0.147
Matt Damon                     0.225      | Shin-ichiro Miki               0.147
Tom Hanks                      0.224      | Toshiyuki Morikawa             0.145
Bruce Willis                   0.224      | Jun Fukuyama                   0.143
Bill Hader                     0.222      | Yuichi Nakamura                0.142
Chris Pratt                    0.221      | Mamoru Miyano                  0.142
Liam Neeson                    0.221      | Ayane Sakura                   0.141

# Plot the closeness centrality vs node degree to see if there is a correlation
closeness_centrality_values = list(closeness_centrality.values())
eigenvector_centrality_values = list(eigenvector_centrality.values())
degree_values = list(dict(G.degree()).values())

fig, axs = plt.subplots(1, 2, figsize=(16, 6))

# plot the closeness centrality vs node degree to see if there is a correlation
axs[0].scatter(degree_values, closeness_centrality_values, alpha=0.5)
axs[0].set_xlabel('Node Degree')
axs[0].set_ylabel('Closeness Centrality')
axs[0].set_title('Closeness Centrality vs Node Degree')

# plot the eigenvector centrality vs node degree to see if there is a correlation
axs[1].scatter(degree_values, eigenvector_centrality_values, alpha=0.5)
axs[1].set_xlabel('Node Degree')
axs[1].set_ylabel('Eigenvector Centrality')
axs[1].set_title('Eigenvector Centrality vs Node Degree')

plt.show()

df_actors_no_japan = df_actors_filtered[df_actors_filtered['birthplace'] != 'Japan']

G_no_japan = create_network(df_actors_no_japan)

# Calculate the closeness centrality of the network
closeness_centrality = nx.closeness_centrality(G_no_japan)
# Calculate the eigenvector centrality of the network
eigenvector_centrality = nx.eigenvector_centrality(G_no_japan)

# Plot the closeness centrality vs node degree to see if there is a correlation
closeness_centrality_values = list(closeness_centrality.values())
eigenvector_centrality_values = list(eigenvector_centrality.values())
degree_values = list(dict(G_no_japan.degree()).values())

fig, axs = plt.subplots(1, 2, figsize=(16, 6))

# plot the closeness centrality vs node degree to see if there is a correlation
axs[0].scatter(degree_values, closeness_centrality_values, alpha=0.5)
axs[0].set_xlabel('Node Degree')
axs[0].set_ylabel('Closeness Centrality')
axs[0].set_title('Closeness Centrality vs Node Degree (without Japan)')

# plot the eigenvector centrality vs node degree to see if there is a correlation
axs[1].scatter(degree_values, eigenvector_centrality_values, alpha=0.5)
axs[1].set_xlabel('Node Degree')
axs[1].set_ylabel('Eigenvector Centrality')
axs[1].set_title('Eigenvector Centrality vs Node Degree (without Japan)')

plt.show()

stemmer = PorterStemmer()

def tokenize_and_clean(text, collocations={}, with_collocations=False): #takes text and dictionary of collocations
    # Exclude symbols and numbers
    text = re.sub(r'\b\d+\b', '', text)
    # Exclude punctuation and convert to lowercase
    text = re.sub(r'\W', ' ', text).lower()

    # Tokenize
    if with_collocations:
        tokenizer = MWETokenizer(list(collocations.keys()))
        tokens = word_tokenize(text)
        tokens = tokenizer.tokenize(tokens)
    else:
        tokens = word_tokenize(text)
    # stem
    tokens = [stemmer.stem(token) for token in tokens if token not in stopwords.words('english')]# TO REMOVE STOPWORDS 
    return tokens

# First clean text
df_movies_filtered['tokens'] = df_movies_filtered['abstract'].apply(lambda x: tokenize_and_clean(x))

# Initialize an empty list to store all bigrams
all_bigrams = []

# For each list of tokens in the 'tokens' column
for tokens in df_movies_filtered['tokens']:
    # Generate the list of bigrams
    bigrams_list = list(bigrams(tokens))
    # Add the bigrams to the list of all bigrams
    all_bigrams.extend(bigrams_list)

all_unique_bigrams = list(set(all_bigrams))
tokens_list = list(df_movies_filtered['tokens'].copy().explode())

# Create a Counter for bigrams and tokens
bigrams_counter = collections.Counter(all_bigrams)
tokens_counter = collections.Counter(tokens_list)

# Create a dictionary to store the data
data = {}

# For each unique pair of words
for w1, w2 in tqdm(all_unique_bigrams):
    # number of times w1 and w2 appear together in all_bigrams
    nii = bigrams_counter[(w1, w2)]
    # number of times w1 appears without w2
    noi = tokens_counter[w1] - nii
    # number of times w2 appears without w1
    nio = tokens_counter[w2] - nii
    # number of times w1 and w2 do not appear together
    noo = len(all_bigrams) - nii - noi - nio

    O = [[nii, nio], [noi, noo]]

    R1 = nii + nio
    C1 = nii + noi
    R2 = noi + noo
    C2 = nio + noo
    N = R1 + R2 + C1 + C2

    E = [[R1*C1/N, R1*C2/N],[R2*C1/N, R2*C2/N]] 

    X_sq = sum((O[i][j] - E[i][j]) ** 2 / E[i][j] for i in range(len(O)) for j in range(len(O[i])))
    p_val = stats.chi2.sf(X_sq, df=1)

    data[(w1, w2)] = O, E, p_val

collocations = {}

for w1, w2 in tqdm(all_unique_bigrams):
    if bigrams_counter[(w1, w2)] > 10 and data[(w1, w2)][2] < 0.001:
        collocations.update({(w1, w2): bigrams_counter[(w1, w2)]})

print(f"\nNumber of collocations: {len(collocations)}")
# Print out the top 20 of them by number of occurrences
sorted(collocations.items(), key=lambda x: x[1], reverse=True)[:10]

# recomputing the tokens with collocations
df_movies_filtered['tokens'] = df_movies_filtered['abstract'].apply(lambda x: tokenize_and_clean(x, collocations, 
                                                                                                       with_collocations=True))

# # save the dataframe as a csv file called df_movies_tokens
# df_movies_filtered.to_csv('data/df_movies_tokens.csv', index=False)

df_movies_tokens = pd.read_csv('data/df_movies_tokens.csv', converters={'tokens': ast.literal_eval, 'genres': ast.literal_eval})

# Compute the best partition using the Louvain method
partition = community_louvain.best_partition(G, resolution=3)

# Compute the modularity of this partition
modularity = community_louvain.modularity(partition, G)

print("Modularity found by Louvain algorithm:", modularity)

# Calculate the number of communities and their sizes
communities = {}
for node, community in partition.items():
    if community not in communities:
        communities[community] = []
    communities[community].append(node)

num_communities = len(communities)
community_sizes = sorted([len(nodes) for nodes in communities.values()], reverse=True)

print("Number of communities:", num_communities)
print("Sizes of communities:", community_sizes)
# Check if the modularity is significantly different than 0
if abs(modularity) > 0.01:  # or any other threshold you consider significant
    print("The modularity is significantly different than 0.")
else:
    print("The modularity is not significantly different than 0.")

# Create a DataFrame from the partition and degree information
df_communities = pd.DataFrame({
    'actor': list(G.nodes),
    'community': [partition[node] for node in G.nodes],
    'degree': [G.degree(node) for node in G.nodes]
})

# Save the dataframe to csv
df_communities.to_csv('data/communities.csv', index=False)

# Load the df_communities csv-file
df_communities = pd.read_csv('data/communities.csv')

# Merge actors_works_df with communities_df
df = pd.merge(df_actors_filtered, df_communities, on='actor')
df = df.drop(columns=["movies"]).explode('movie_IDs')


# Merge with abstracts_df
df = pd.merge(df, df_movies_tokens, left_on='movie_IDs', right_on='movie_ID')

# # Get abstract tokens for all communities
all_communities_abstracts = df.groupby('community')['tokens'].agg("sum").reset_index()

all_communities_abstracts.columns = ['Community', 'Abstract Tokens']

# Merge df_actors_filtered with communities df
df_actors_communities = pd.merge(df_actors_filtered, df_communities, on='actor')

# For community in top20_communities, calculate community attributes
# Extract top 20 communities
top20_communities_idx = df_communities['community'].value_counts().nlargest(45).index
top20_community_actors = df_actors_communities[df_actors_communities['community'].isin(top20_communities_idx)]

# Calculate in each community (group_by)
def average_of_list(x):
    return np.mean([val for sublist in x for val in sublist])
def sum_dicts(dicts):
    result = {}
    for d in dicts:
        for k, v in d.items():
            if k in result:
                result[k] += v
            else:
                result[k] = v
    return result
top20_community_attributes = top20_community_actors.groupby('community').agg({
    'actor': 'count',
    'age': 'mean',
    'ratings': [('mean', average_of_list)],
    'popularity': 'mean',
    'degree': [('mean', 'mean')],
    'birthplace': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
})

# Clean columns
top20_community_attributes.columns = ['_'.join(col).strip() for col in top20_community_attributes.columns.values]
top20_community_attributes.reset_index(inplace=True)

# Calculate main genre in each community
df_grouped = df_actors_communities.groupby('community')['genres'].apply(sum_dicts).reset_index()
df_grouped = df_grouped.dropna()
df_grouped = df_grouped.loc[df_grouped.groupby('community')['genres'].idxmax()]
top20_community_attributes = pd.merge(top20_community_attributes, df_grouped, on='community', how='inner')
top20_community_attributes = top20_community_attributes.drop(columns=['genres'])

# Rename columns
top20_community_attributes = top20_community_attributes.rename(columns={'birthplace_<lambda>': 'birthplace'})
top20_community_attributes = top20_community_attributes.rename(columns={'level_1': 'main_genre'})

# Create a success column that sums popularity and rating means
rating_min = top20_community_attributes['ratings_mean'].min()
rating_max = top20_community_attributes['ratings_mean'].max()
popularity_min = top20_community_attributes['popularity_mean'].min()
popularity_max = top20_community_attributes['popularity_mean'].max()

rating_standardized = ((top20_community_attributes['ratings_mean'] - rating_min)/(rating_max - rating_min))
popularity_standardized = ((top20_community_attributes['popularity_mean'] - popularity_min)/(popularity_max - popularity_min))

top20_community_attributes['success'] = rating_standardized * popularity_standardized

# top20_community_attributes.sort_values(by='ratings_mean', ascending=False)

# Get the top 9 communities and filter the abstracts for these communities
# top9_communities = df_communities['community'].value_counts().nlargest(9).index
top9_communities_idx = top20_community_attributes['ratings_mean'].nlargest(9).index

top9_communities = top20_community_attributes.iloc[top9_communities_idx]['community']

# find the top 9 community abstracts
top9_communities_abstracts = all_communities_abstracts[all_communities_abstracts['Community'].isin(top9_communities)]

top_terms = {}
top_tfidf_terms = {}
all_tfidf_terms = {}

# (for less computational cost) get IDF for each term once and store the results
idf_dict = {}
for term in tqdm(set.union(*top9_communities_abstracts['Abstract Tokens'].apply(set))):
    idf_dict[term] = math.log(len(top9_communities_abstracts) / 
                              sum(term in abstract for abstract in top9_communities_abstracts['Abstract Tokens']))


for community in tqdm(top9_communities):
    # Get abstract tokens for the community
    abstract_tokens = top9_communities_abstracts[top9_communities_abstracts['Community'] == community]['Abstract Tokens'].values[0]
    
    # The top 10 TF terms
    term_counts = Counter(abstract_tokens)
    top_terms[community] = term_counts.most_common(10)


    tfidf = {}
    for term, count in term_counts.items():
        # TF for all terms in top 9 communities
        tf = count / len(abstract_tokens)

        # Get IDF from the precalculated dictionary
        idf = idf_dict[term]
        
        # TF-IDF for all terms in top 9 communities
        tfidf[term] = tf * idf

    all_tfidf_terms[community] = sorted(tfidf.items(), key=lambda x: x[1], reverse=True)
    
    # Get the top 10 TF-IDF words
    top_tfidf_terms[community] = sorted(tfidf.items(), key=lambda x: x[1], reverse=True)[:10]

for item in top_tfidf_terms.items():
    print(f"Community {item[0]}:")
    print(', '.join([term[0] for term in item[1]]))

top9_communities_df = df_communities[df_communities['community'].isin(top9_communities)]

# Group by Community and Actor, and sum the Degree to get the total Degree for each Actor in each Community
grouped_df = top9_communities_df.groupby(['community', 'actor'])['degree'].sum().reset_index()

# Make new dataframe storing the top 5 actors by degree for each community
top_actors = grouped_df.groupby('community').apply(lambda x: x.nlargest(5, 'degree')).reset_index(drop=True)

# Save all_tfidf_terms as a JSON file in the data folder
with open('data/all_tfidf_terms.json', 'w') as f:
    json.dump(all_tfidf_terms, f)

# Save top9_communities as a CSV file in the data folder
pd.DataFrame(top9_communities, columns=['community']).to_csv('data/top9_communities.csv', index=False)

# Save top_actors as a CSV file in the data folder
top_actors.to_csv('data/top_actors.csv', index=False)

# load files for wordcloud
top_actors = pd.read_csv('data/top_actors.csv')

top9_communities = pd.read_csv('data/top9_communities.csv')

with open('data/all_tfidf_terms.json', 'r') as file:
    all_tfidf_terms = json.load(file)

fig, axs = plt.subplots(3, 3, figsize=(24, 24))

for i, community in enumerate(top9_communities["community"]):
    # Get the already calculated top TF-IDF terms for the community
    tfidf_terms = dict(all_tfidf_terms[str(community)])
    
    # Create the word cloud
    wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='white', 
                stopwords = None, 
                min_font_size = 10).generate_from_frequencies(tfidf_terms)
    
    # Subplot indices
    row = i // 3
    col = i % 3

    # Plot the word cloud
    axs[row, col].imshow(wordcloud)
    axs[row, col].axis("off")

    # Get the names of the top three actors in the community
    top_three_actors = top_actors[top_actors['community'] == community]['actor'].values
    # Create the title string with newlines
    title_string = "\n".join(top_three_actors)

    # Set title of subplot to the names of the top three actors, each on a new line
    axs[row, col].set_title(f"Top actors in community {community}:\n{title_string}", fontsize=15)

# Remove any extra subplots
for j in range(i+1, 9):
    fig.delaxes(axs.flatten()[j])

plt.tight_layout(pad = 1) 
plt.subplots_adjust(hspace = 0.25)  # Increase the hspace value
plt.show()

# Save the word cloud to png
# fig.savefig('wordcloud.png')

# load wordcloud
Image(filename='figures/wordcloud.png')

# Create HTML string for images
html_str = """
<div style="display: flex; justify-content: space-around; align-items: flex-start;">
    <div>
        <img src='figures/marvel.png' style='width: 250px;'>
        <p style="text-align: center;">Community 8</p>
    </div>
    <div>
        <img src='figures/anime_dj.png' style='width: 250px;'>
        <p style="text-align: center;">Community 43</p>
    </div>
    <div>
        <img src='figures/harry_potter.png' style='width: 250px;'>
        <p style="text-align: center;">Community 57</p>
    </div>
</div>
"""

# Display the images
display(HTML(html_str))

fig, ax = plt.subplots(3,3, figsize=(16, 18))

metrics = ['ratings_mean', 'popularity_mean', 'success']
metrics_map = {'ratings_mean': 'community rating', 'popularity_mean': 'community popularity', 'success': 'community success'}
x_values = ['age_mean', 'main_genre', 'birthplace']
titles = ['Community Age vs ', 'Community Genre vs ', 'Community birthplace vs ']
colors = ['blue', 'green', 'red']  # List of colors for each row

for i, metr in enumerate(metrics):
    for j, x_val in enumerate(x_values):
        ax[i,j].set_title(titles[j] + metrics_map[metr])
        
        # Use scatter plots for 'age_mean' and point plots for the rest
        if x_val == 'age_mean':
            sns.regplot(x=top20_community_attributes[x_val], y=top20_community_attributes[metr], color=colors[i], ax=ax[i,j], ci=95)
            ax[i,j].set_xlabel('Mean Age')
            ax[i,j].set_ylabel(metrics_map[metr])
            
            # Calculate Spearman's correlation
            corr, _ = stats.spearmanr(top20_community_attributes[x_val], top20_community_attributes[metr])
            ax[i,j].text(0.1, 0.9, f'Spearman: {corr:.2f}', transform=ax[i,j].transAxes)
        else:
            order = top20_community_attributes.groupby(x_val)[metr].mean().sort_values().index
            sns.pointplot(x=x_val, y=metr, data=top20_community_attributes, order=order, capsize=.1, linestyle='none', ax=ax[i,j], color=colors[i])
            ax[i,j].set_ylabel(metrics_map[metr])

        # Rotate x-axis labels for each subplot
        for label in ax[i,j].get_xticklabels():
            label.set_rotation(60)
        
        # Adjust age ticks for the first column of subplots
        if j == 0:
            start, end = ax[i,j].get_xlim()
            start, end = int(round(start)), int(round(end))
            ax[i,j].xaxis.set_ticks(np.arange(start, end, 5))
        
        # Add grid to all plots
        ax[i,j].grid(True)

# add vertical space
fig.subplots_adjust(hspace=0.5)

plt.show()

# Save the figure to a png file
# fig.savefig('figures/community_metrics.png')

# Load community metrics plot
Image(filename='figures/community_metrics.png')

Project Assignment B¶

1. Motivation¶

2. Basic stats¶

Data retrieval and cleaning/preprocessing¶

Importing libraries¶

Functions used to make API calls¶

Retrieve actor information (name, id, gender, birthday, birthplace, popularity)¶

Creating the Actor Dataframe and initial preprocessing¶

Retrieve movie titles and IDs¶

Collect movie information trough API (rating, popularity, genre, release date, abstract)¶

Initial preprocessing of Movies Dataframe¶

Remove old and future movies¶

Cleaning the birthplace column, to avoid having the same country spelled differently and removing symbols¶

Finding the main genre for the popular actors¶

Actor rating¶

Saving the final dataframes¶

Taking a look into the data of interest, distributions and correlations¶

Investigate categorical attributes¶

Creating the network¶

3. Tools, theory and analysis¶

Degrees and weights¶

Plot of degree distribution¶

Most collaborative pairs in the network¶

Determining the regime of the network¶

Plot the network according to birthplace¶

Calculate assortativity¶

Calculating the closeness and eigenvector centrality¶

Plotting the degree vs centrality¶

Textual Data Examination and Analysis¶

Partitioning of network into communities¶

Community data collecting¶

TF-IDF for wordclouds¶

Wordcloud analysis¶

What distinguishes well performing communities?¶

4. Discussion¶

What went well?¶

What is still missing? What could be improved? Why?¶